import asyncio

from com.arcfox.base.base_spider import ExecuteType
from com.arcfox.manager.redis_task_manager import RedisTaskManager
from com.arcfox.school.processor.yzy.yzy_major_ranking_processor import YZYMajorRankingProcessor
from com.arcfox.school.spider.yzy.yzy_base_spider import YZYBaseSpider
from com.arcfox.util import async_request as requests
from com.arcfox.util.redis_key_manager import YZY_MAJOR_RANKING_TASK_KEY
from com.arcfox.util.util import random_sleep


class YZYMajorRankSpider(YZYBaseSpider):
    def __init__(self):
        super().__init__()
        self.url = "https://uwf7de983aad7a717eb.youzy.cn/youzy.dms.basiclib.api.eduranking.mr.search"
        self.start_page = 1
        self.years = [2020, 2021, 2022]
        self.task_manager = RedisTaskManager(YZY_MAJOR_RANKING_TASK_KEY)
        self.processor = YZYMajorRankingProcessor()

    def get_params(self, major_code, year, page_num):
        return '{"year":"' + str(year) + '","majorCode":"' + str(
            major_code) + '","keyword":"","collegeProvinceCodes":[],"features":[],' \
                          '"collegeCategories":[],"natureTypes":[],"pageIndex":' + str(page_num) + ',"pageSize":20}'

    def get_headers(self, params):
        return {
            "Host": "uwf7de983aad7a717eb.youzy.cn",
            "u-sign": self.get_u_sgin(params),
            "Content-Type": "application/json",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
            "Referer": "https://pv4y-pc.youzy.cn/",
        }

    async def _pull_task(self):
        return await self.task_manager.pull_tasks(5), ExecuteType.FINISH

    async def _crawl_by_task(self, tasks):
        for task in tasks:
            params = self.get_params(task['major_code'], task['year'], self.start_page)
            resp = await requests.post_with_session(url=self.url, data=params, headers=self.get_headers(params))
            if resp.code == 200:
                result_json = resp.json()
                await self.processor.parse_and_save_data(task, resp.json())
                total_count = result_json['result']["totalCount"]
                total_pages = total_count / 20 if total_count % 20 == 0 else total_count / 20 + 1
                for i in range(2, int(total_pages) + 1):
                    params_item = self.get_params(task['major_code'], task['year'], i)
                    resp_item = await requests.post_with_session(url=self.url, data=params_item,
                                                                 headers=self.get_headers(params_item))
                    if resp_item.code == 200:
                        await self.processor.parse_and_save_data(task, resp_item.json())
                    else:
                        await self.task_manager.add_fail_tasks(task)
                    await random_sleep(0.2)
            else:
                await self.task_manager.add_fail_tasks(task)


if __name__ == "__main__":
    asyncio.run(YZYMajorRankSpider().start_crawl())
